Sentence Take #1 - A Sequence of Words


In [2]:
import re
import reprlib
RE_WORD = re.compile('\w+')

class Sentence:
    
    def __init__(self, text):
        self.text = text
        self.words = RE_WORD.findall(text)
        
    def __getitem__(self, index):
        return self.words[index]
    
    def __len__(self):
        return len(self.words)
    
    def __repr__(self):
        return 'Sentence(%s)' % reprlib.repr(self.text)

In [3]:
s = Sentence('"The time has come, " the Walrus said,')
s


Out[3]:
Sentence('"The time ha... Walrus said,')

In [4]:
for word in s:
    print(word)


The
time
has
come
the
Walrus
said

In [5]:
list(s)


Out[5]:
['The', 'time', 'has', 'come', 'the', 'Walrus', 'said']

In [6]:
s[0]


Out[6]:
'The'

In [7]:
s[5]


Out[7]:
'Walrus'

In [8]:
s[-1]


Out[8]:
'said'

Why Sequences Are Iterable: The iter Function


In [9]:
class Foo:
    def __iter__(self):
        pass

In [10]:
from collections import abc

In [11]:
issubclass(Foo, abc.Iterable)


Out[11]:
True

In [12]:
f = Foo()
isinstance(f, abc.Iterable)


Out[12]:
True

In [13]:
issubclass(Sentence, abc.Iterable)


Out[13]:
False

Iterables Versus Iterators


In [14]:
s = 'ABC'
for char in s:
    print(char)


A
B
C

In [15]:
s = 'ABC'
it = iter(s)
while True:
    try:
        print(next(it))
    except StopIteration:
        del it
        break


A
B
C

In [16]:
s3 = Sentence('Pig and Pepper')
it = iter(s3)
it


Out[16]:
<iterator at 0x98cc37b5f8>

In [17]:
next(it)


Out[17]:
'Pig'

In [18]:
next(it)


Out[18]:
'and'

In [19]:
next(it)


Out[19]:
'Pepper'

In [20]:
next(it)


---------------------------------------------------------------------------
StopIteration                             Traceback (most recent call last)
<ipython-input-20-2cdb14c0d4d6> in <module>()
----> 1 next(it)

StopIteration: 

In [21]:
list(it)


Out[21]:
[]

In [22]:
list(iter(s3))


Out[22]:
['Pig', 'and', 'Pepper']

Sentence Take #2: A Classic Iterator


In [23]:
import re
import reprlib
RE_WORD = re.compile('\w+')

class SentenceIterator:
    
    def __init__(self, words):
        self.words = words
        self.index = 0
        
    def __next__(self):
        try:
            word = self.words[self.index]
        except IndexError:
            raise StopIteration()
        self.index += 1
        return word
    
    def __iter__(self):
        return self
    
    
class Sentence:
    
    def __init__(self, text):
        self.text = text
        self.words = RE_WORD.findall(text)
        
    def __repr__(self):
        return 'Sentence(%s)' % reprlib.repr(self.text)
    
    def __iter__(self):
        return SentenceIterator(self.words)

In [24]:
s = Sentence('"The time has come, " the Walrus said,')
s


Out[24]:
Sentence('"The time ha... Walrus said,')

In [25]:
for word in s:
    print(word)


The
time
has
come
the
Walrus
said

In [26]:
list(s)


Out[26]:
['The', 'time', 'has', 'come', 'the', 'Walrus', 'said']

Sentence Take #3: A Generator Function


In [27]:
import re
import reprlib
RE_WORD = re.compile('\w+')

class Sentence:
    
    def __init__(self, text):
        self.text = text
        self.words = RE_WORD.findall(text)
        
    def __repr__(self):
        return 'Sentence(%s)' % reprlib.repr(self.text)
    
    def __iter__(self):
        for word in self.words:
            yield word
        return

How a Generator Function Works


In [28]:
def gen_123():
    yield 1
    yield 2
    yield 3

In [29]:
gen_123


Out[29]:
<function __main__.gen_123>

In [30]:
gen_123()


Out[30]:
<generator object gen_123 at 0x00000098CC3CA678>

In [33]:
for i in gen_123():
    print(i)


1
2
3

In [34]:
g = gen_123()
next(g)


Out[34]:
1

In [35]:
next(g)


Out[35]:
2

In [36]:
next(g)


Out[36]:
3

In [37]:
next(g)


---------------------------------------------------------------------------
StopIteration                             Traceback (most recent call last)
<ipython-input-37-5f315c5de15b> in <module>()
----> 1 next(g)

StopIteration: 

In [38]:
def gen_AB():
    print('start')
    yield 'A'
    print('continue')
    yield 'B'
    print('end.')

In [39]:
for c in gen_AB():
    print('-->', c)


start
--> A
continue
--> B
end.

Sentence Take #4: A Lazy Implementation


In [40]:
import re
import reprlib
RE_WORD = re.compile('\w+')

class Sentence:
    
    def __init__(self, text):
        self.text = text
        
    def __repr__(self):
        return 'Sentence(%s)' % reprlib.repr(self.text)
    
    def __iter__(self):
        for match in RE_WORD.finditer(self.text):
            yield match.group()

Sentence Take #5: A Generator Expression


In [41]:
res1 = [x*3 for x in gen_AB()]


start
continue
end.

In [42]:
for i in res1:
    print('-->', i)


--> AAA
--> BBB

In [45]:
res2 = (x*3 for x in gen_AB())

In [46]:
res2


Out[46]:
<generator object <genexpr> at 0x00000098CC3CA308>

In [47]:
for i in res2:
    print('-->', i)


start
--> AAA
continue
--> BBB
end.

In [48]:
import re
import reprlib
RE_WORD = re.compile('\w+')

class Sentence:
    
    def __init__(self, text):
        self.text = text
        
    def __repr__(self):
        return 'Sentence(%s)' % reprlib.repr(self.text)
    
    def __iter__(self):
        return (match.group() for match in RE_WORD.finditer(self.text))

Another Example: Arithmetic Progression Generator


In [49]:
class ArithmeticProgression:
    
    def __init__(self, begin, step, end=None):
        self.begin = begin
        self.step = step
        self.end = end
        
    def __iter__(self):
        result = type(self.begin + self.step)(self.begin)
        forever = self.end is None
        index = 0
        while forever or result < self.end:
            yield result
            index += 1
            result = self.begin + self.step * index

In [50]:
ap = ArithmeticProgression(0, 1, 3)

In [51]:
list(ap)


Out[51]:
[0, 1, 2]

In [52]:
ap = ArithmeticProgression(1, .5, 3)

In [53]:
list(ap)


Out[53]:
[1.0, 1.5, 2.0, 2.5]

In [54]:
ap = ArithmeticProgression(0, 1/3, 1)
list(ap)


Out[54]:
[0.0, 0.3333333333333333, 0.6666666666666666]

In [55]:
from fractions import Fraction
ap = ArithmeticProgression(0, Fraction(1, 3), 1)
list(ap)


Out[55]:
[Fraction(0, 1), Fraction(1, 3), Fraction(2, 3)]

In [56]:
from decimal import Decimal
ap = ArithmeticProgression(0, Decimal('.1'), .3)
list(ap)


Out[56]:
[Decimal('0'), Decimal('0.1'), Decimal('0.2')]

In [61]:
def aritprog_gen(begin, step, end=None):
    result = type(begin + step)(begin)
    forever = end is None
    index = 0
    while forever or result < end:
        yield result
        index += 1
        result = begin + step * index

In [62]:
from decimal import Decimal
ap = aritprog_gen(0, Decimal('.1'), .3)
list(ap)


Out[62]:
[Decimal('0'), Decimal('0.1'), Decimal('0.2')]

Arithmetic Progression with itertools


In [63]:
import itertools
gen = itertools.count(1, .5)
next(gen)


Out[63]:
1

In [64]:
next(gen)


Out[64]:
1.5

In [65]:
next(gen)


Out[65]:
2.0

In [66]:
next(gen)


Out[66]:
2.5

In [67]:
gen = itertools.takewhile(lambda n: n < 3, itertools.count(1, .5))
list(gen)


Out[67]:
[1, 1.5, 2.0, 2.5]

In [68]:
import itertools

In [69]:
def aritprog_gen(begin, step, end=None):
    first = type(begin + step)(begin)
    ap_gen = itertools.count(first, step)
    if end is not None:
        ap_gen = itertools.takewhile(lambda: n < end, ap_gen)
    return ap_gen

Generator Functions in the Standard Library

Filtering generator functions


In [70]:
def vowel(c):
    return c.lower() in 'aeiou'

In [71]:
list(filter(vowel, 'Aardvark'))


Out[71]:
['A', 'a', 'a']

In [72]:
import itertools

In [73]:
list(itertools.filterfalse(vowel, 'Aardvark'))


Out[73]:
['r', 'd', 'v', 'r', 'k']

In [74]:
list(itertools.dropwhile(vowel, 'Aardvark'))


Out[74]:
['r', 'd', 'v', 'a', 'r', 'k']

In [75]:
list(itertools.takewhile(vowel, 'Aardvark'))


Out[75]:
['A', 'a']

In [76]:
list(itertools.compress('Aardvark', (1,0,1,1,0,1)))


Out[76]:
['A', 'r', 'd', 'a']

In [78]:
list(itertools.islice('Aardvark', 4))


Out[78]:
['A', 'a', 'r', 'd']

In [79]:
list(itertools.islice('Aardvark', 4, 7))


Out[79]:
['v', 'a', 'r']

In [80]:
list(itertools.islice('Aardvark', 1, 7, 2))


Out[80]:
['a', 'd', 'a']

Mapping generator functions


In [81]:
sample = [5, 4, 2, 8, 7, 6, 3, 0, 9, 1]
import itertools
list(itertools.accumulate(sample))


Out[81]:
[5, 9, 11, 19, 26, 32, 35, 35, 44, 45]

In [82]:
list(itertools.accumulate(sample, min))


Out[82]:
[5, 4, 2, 2, 2, 2, 2, 0, 0, 0]

In [83]:
list(itertools.accumulate(sample, max))


Out[83]:
[5, 5, 5, 8, 8, 8, 8, 8, 9, 9]

In [84]:
import operator
list(itertools.accumulate(sample, operator.mul))


Out[84]:
[5, 20, 40, 320, 2240, 13440, 40320, 0, 0, 0]

In [85]:
list(itertools.accumulate(range(1, 11), operator.mul))


Out[85]:
[1, 2, 6, 24, 120, 720, 5040, 40320, 362880, 3628800]

In [87]:
print(list(enumerate('albatroz', 1)))


[(1, 'a'), (2, 'l'), (3, 'b'), (4, 'a'), (5, 't'), (6, 'r'), (7, 'o'), (8, 'z')]

In [88]:
import operator
list(map(operator.mul, range(11), range(11)))


Out[88]:
[0, 1, 4, 9, 16, 25, 36, 49, 64, 81, 100]

In [89]:
list(map(operator.mul, range(11), [2, 4, 8]))


Out[89]:
[0, 4, 16]

In [90]:
list(map(lambda a, b: (a, b), range(11), [2, 4, 8]))


Out[90]:
[(0, 2), (1, 4), (2, 8)]

In [91]:
import itertools
list(itertools.starmap(operator.mul, enumerate('albatroz', 1)))


Out[91]:
['a', 'll', 'bbb', 'aaaa', 'ttttt', 'rrrrrr', 'ooooooo', 'zzzzzzzz']

In [95]:
sample = [5, 4, 2, 8, 7, 6, 3, 0, 9, 1]
print(list(itertools.starmap(lambda a, b: b/a,enumerate(itertools.accumulate(sample), 1))))


[5.0, 4.5, 3.6666666666666665, 4.75, 5.2, 5.333333333333333, 5.0, 4.375, 4.888888888888889, 4.5]

Generator functions that merge multiple input iterables


In [96]:
list(itertools.chain('ABC', range(2)))


Out[96]:
['A', 'B', 'C', 0, 1]

In [97]:
list(itertools.chain(enumerate('ABC')))


Out[97]:
[(0, 'A'), (1, 'B'), (2, 'C')]

In [98]:
list(itertools.chain.from_iterable(enumerate('ABC')))


Out[98]:
[0, 'A', 1, 'B', 2, 'C']

In [99]:
list(zip('ABC', range(5)))


Out[99]:
[('A', 0), ('B', 1), ('C', 2)]

In [100]:
list(zip('ABC', range(5), [10, 20, 30, 40]))


Out[100]:
[('A', 0, 10), ('B', 1, 20), ('C', 2, 30)]

In [102]:
list(itertools.zip_longest('ABC', range(5)))


Out[102]:
[('A', 0), ('B', 1), ('C', 2), (None, 3), (None, 4)]

In [103]:
list(itertools.zip_longest('ABC', range(5), fillvalue='?'))


Out[103]:
[('A', 0), ('B', 1), ('C', 2), ('?', 3), ('?', 4)]

In [1]:
import itertools

In [2]:
list(itertools.product('ABC', range(2)))


Out[2]:
[('A', 0), ('A', 1), ('B', 0), ('B', 1), ('C', 0), ('C', 1)]

In [4]:
suits = 'spades hearts diamonds clubs'.split()
print(list(itertools.product('AK', suits)))


[('A', 'spades'), ('A', 'hearts'), ('A', 'diamonds'), ('A', 'clubs'), ('K', 'spades'), ('K', 'hearts'), ('K', 'diamonds'), ('K', 'clubs')]

In [5]:
list(itertools.product('ABC'))


Out[5]:
[('A',), ('B',), ('C',)]

In [7]:
print(list(itertools.product('ABC',repeat = 2)))


[('A', 'A'), ('A', 'B'), ('A', 'C'), ('B', 'A'), ('B', 'B'), ('B', 'C'), ('C', 'A'), ('C', 'B'), ('C', 'C')]

In [8]:
print(list(itertools.product(range(2), repeat=3)))


[(0, 0, 0), (0, 0, 1), (0, 1, 0), (0, 1, 1), (1, 0, 0), (1, 0, 1), (1, 1, 0), (1, 1, 1)]

In [9]:
rows = itertools.product('AB', range(2), repeat=2)
for row in rows: print(row)


('A', 0, 'A', 0)
('A', 0, 'A', 1)
('A', 0, 'B', 0)
('A', 0, 'B', 1)
('A', 1, 'A', 0)
('A', 1, 'A', 1)
('A', 1, 'B', 0)
('A', 1, 'B', 1)
('B', 0, 'A', 0)
('B', 0, 'A', 1)
('B', 0, 'B', 0)
('B', 0, 'B', 1)
('B', 1, 'A', 0)
('B', 1, 'A', 1)
('B', 1, 'B', 0)
('B', 1, 'B', 1)

Generator functions that expand each input item into multiple output items


In [10]:
ct = itertools.count()
next(ct)


Out[10]:
0

In [11]:
next(ct), next(ct), next(ct)


Out[11]:
(1, 2, 3)

In [12]:
list(itertools.islice(itertools.count(1, .3), 3))


Out[12]:
[1, 1.3, 1.6]

In [13]:
cy = itertools.cycle('ABC')
next(cy)


Out[13]:
'A'

In [14]:
list(itertools.islice(cy, 7))


Out[14]:
['B', 'C', 'A', 'B', 'C', 'A', 'B']

In [15]:
rp = itertools.repeat(7)
next(rp), next(rp)


Out[15]:
(7, 7)

In [16]:
list(itertools.repeat(8, 4))


Out[16]:
[8, 8, 8, 8]

In [17]:
import operator
list(map(operator.mul, range(11), itertools.repeat(5)))


Out[17]:
[0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50]

In [18]:
list(itertools.combinations('ABC', 2))


Out[18]:
[('A', 'B'), ('A', 'C'), ('B', 'C')]

In [19]:
list(itertools.combinations_with_replacement('ABC', 2))


Out[19]:
[('A', 'A'), ('A', 'B'), ('A', 'C'), ('B', 'B'), ('B', 'C'), ('C', 'C')]

In [20]:
list(itertools.permutations('ABC', 2))


Out[20]:
[('A', 'B'), ('A', 'C'), ('B', 'A'), ('B', 'C'), ('C', 'A'), ('C', 'B')]

In [22]:
print(list(itertools.product('ABC', repeat=2)))


[('A', 'A'), ('A', 'B'), ('A', 'C'), ('B', 'A'), ('B', 'B'), ('B', 'C'), ('C', 'A'), ('C', 'B'), ('C', 'C')]

Rearranging generator functions


In [23]:
list(itertools.groupby('LLLLAAGGG'))


Out[23]:
[('L', <itertools._grouper at 0x45be573908>),
 ('A', <itertools._grouper at 0x45be573fd0>),
 ('G', <itertools._grouper at 0x45be573b00>)]

In [24]:
for char, group in itertools.groupby('LLLLAAGGG'):
    print(char, '->', list(group))


L -> ['L', 'L', 'L', 'L']
A -> ['A', 'A']
G -> ['G', 'G', 'G']

In [25]:
animals = ['duck', 'eagle', 'rat', 'giraffe', 'bear', 'bat', 'dolphin', 'shark', 'lion']
animals.sort(key=len)
animals


Out[25]:
['rat', 'bat', 'duck', 'bear', 'lion', 'eagle', 'shark', 'giraffe', 'dolphin']

In [27]:
for length, group in itertools.groupby(animals, len):
    print(length, '->', list(group))


3 -> ['rat', 'bat']
4 -> ['duck', 'bear', 'lion']
5 -> ['eagle', 'shark']
7 -> ['giraffe', 'dolphin']

In [28]:
for length, group in itertools.groupby(reversed(animals), len):
    print(length, '->', list(group))


7 -> ['dolphin', 'giraffe']
5 -> ['shark', 'eagle']
4 -> ['lion', 'bear', 'duck']
3 -> ['bat', 'rat']

In [29]:
list(itertools.tee('ABC'))


Out[29]:
[<itertools._tee at 0x45be606908>, <itertools._tee at 0x45be606948>]

In [36]:
g1, g2 = itertools.tee('ABC')

In [37]:
next(g1)


Out[37]:
'A'

In [38]:
next(g2)


Out[38]:
'A'

In [39]:
next(g2)


Out[39]:
'B'

In [40]:
list(g1)


Out[40]:
['B', 'C']

In [41]:
list(g2)


Out[41]:
['C']

In [42]:
list(zip(*itertools.tee('ABC')))


Out[42]:
[('A', 'A'), ('B', 'B'), ('C', 'C')]

New Syntax in Python 3.3: yield from


In [47]:
def chain(*iterables):
    for it in iterables:
        for i in it:
            yield i

In [48]:
s = 'ABC'
t = tuple(range(3))
list(chain(s, t))


Out[48]:
['A', 'B', 'C', 0, 1, 2]

In [49]:
def chain(*iterables):
    for i in iterables:
        yield from i

In [50]:
list(chain(s, t))


Out[50]:
['A', 'B', 'C', 0, 1, 2]

Iterable Reducing Functions


In [51]:
all([1, 2, 3])


Out[51]:
True

In [52]:
all([1, 0, 3])


Out[52]:
False

In [53]:
all([])


Out[53]:
True

In [54]:
any([1, 2, 3])


Out[54]:
True

In [55]:
any([1, 0, 3])


Out[55]:
True

In [56]:
any([0, 0, 0])


Out[56]:
False

In [57]:
any([])


Out[57]:
False

In [58]:
g = (n for n in [0, 0.0, 7, 8])
any(g)


Out[58]:
True

In [59]:
next(g)


Out[59]:
8

A Closer Look at the iter Function


In [63]:
from random import randint
def d6():
    return randint(1, 6)

In [72]:
d6_iter = iter(d6, 1)
d6_iter


Out[72]:
<callable_iterator at 0x45be61b438>

In [73]:
for roll in d6_iter:
    print(roll)


4
2
3

In [ ]: